package com.terren.spider.core.html.processor;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.commons.collections.CollectionUtils;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.scheme.ICustomScheme;
import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.util.common.BeanUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 用于抓取列表页链接的processor 抓取并分析列表页面， 并找到所有目标页面的链接 放到缓存中
 * 
 * @author chenyu
 *
 */
public class HtmlArticleSearchUrlProcessor extends BasicHtmlProcessor implements PageProcessor {

	protected List<ArticleSearch> articleList = new ArrayList<ArticleSearch>();// 存放搜索页的标题，链接，发布时间等信息

	public HtmlArticleSearchUrlProcessor(Configuration configuration) throws Exception {
		super(configuration);
	}

	@Override
	public void process(Page page) {
		logMessage(page);
		if (isExcludeTargetUrl(page, configuration.getEntry())||super.isTargetUrl(page, configuration.getEntry())) {// 被忽略的目标页面集合， 如 搜索页面
			Integer recordLimit = configuration.getEntry().getRecordLimit() != null ? configuration.getEntry().getRecordLimit() : SysConfig.RECORD_LIMIT;
			Integer recordSize = configuration.getPageRecordSize() != null ? configuration.getPageRecordSize() : SysConfig.RECORD_SIZE;

			/**
			 * ************* 添加列表页URL begin ********
			 */
			List<String> requestUrl = new ArrayList<>();

			if (isContainHelpUrl(configuration.getEntry())) {// 如果有列表页
				requestUrl.addAll(this.findHelpUrls(page, configuration.getEntry()));// 添加列表页
			}
			for (ICustomScheme scheme : configuration.getSchemes()) {// 自定义列表页方案。等
				if (BeanUtil.includeMethod("findHelpUrl", scheme.getClass())) {
					Set<String> urls = scheme.findHelpUrl(page, configuration.getEntry());
					log.info("[entry:"+configuration.getEntry().getEntryId()+"]"+"[scheme:"+scheme.getClass().getName());
					if (null != urls) {
						requestUrl.addAll(urls);
					}
					log.info("[method:findHelpUrl]"+"[size:"+urls.size()+"]");
				}
			}

			if (getHelpPagingUrlCount() < ((recordLimit + SysConfig.MORE_SIZE) / recordSize - 1)) {// 分页数量
				requestUrl.addAll(this.findHelpPaginationUrl(page, configuration));// 分页
			}
			helpUrls.addAll(requestUrl);
			super.addTargetRequests(page, helpUrls);// 把分页、列表页加入到扒取队列
			/**
			 * ************* 添加列表页URL end ********
			 */

			/**
			 * ************* 添加目标页URL begin ********
			 */

			for (ICustomScheme scheme : configuration.getSchemes()) {// 自定义列表页方案。等
				if (CollectionUtils.isEmpty(scheme.findTitleInfo(page, configuration.getEntry()))) {// 如果没有自定义获取url方案
					this.findTargetUrls(page, configuration.getEntry());// 把列表页内的所有目标页找到， 放到set中 使用查找标题
				}
				/**
				 * 从自定义方案中找到URL、标题、时间、来源等信息
				 */
				else {
					List<ArticleSearch> findItems = scheme.findTitleInfo(page, configuration.getEntry());
					for (ArticleSearch articleSearch : findItems) {
						targetUrls.add(articleSearch.getUrl());
					}
					articleList.addAll(findItems);
				}
			}

			/**
			 * ************* 添加目标页URL end ********
			 */
		} else {
			targetUrls.add(getCurrentPageUrl(page));
		}
	}

	public List<ArticleSearch> getArticleList() {
		return articleList;
	}

	public void setArticleList(List<ArticleSearch> articleList) {
		this.articleList = articleList;
	}
}
